This is an optional step. You can skip it and install packages to your current environment.
python -m venv .venv/histtest
source .venv/histtest/bin/activate
pip install \
numpy==1.19.5 \
matplotlib==3.0.3 \
jupyter==1.0.0 \
pillow==5.4.1 \
scikit-image==0.14.2 \
pycocotools==2.0.3 \
himpy=0.0.1
import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as image_utils
from matplotlib.collections import PatchCollection
from matplotlib.patches import Polygon, Rectangle
%matplotlib inline
# import skimage.io as io
import skimage.draw as draw
from himpy.histogram import operations
from himpy.executor import Parser, Evaluator
from himpy.utils import E
%load_ext autoreload
%autoreload 2
import sys
sys.path.insert(0, "../")
from utils.datasets import COCOLoader
# feature extraction
from utils.feature_extraction import (
FeatureMerger,
PositionSetTransformer,
filter_data,
create_histogram,
create_histogram_,
extract_elements,
extract_element_set
)
from utils.feature_extraction.coco import COCOObjectSetTransformer, load_coco_histograms
# search engine
from utils.search_engine import SearchEngine
# plot
from utils.plot.matplotlib_plot import (
plot_position_grid,
plot_object_ids,
show_operation_result,
show_retrieved_images
)
The COCO dataset, website:
loader = COCOLoader()
# Download the datasets
coco = loader.fetch_load()
loading annotations into memory... Done (t=0.67s) creating index... index created!
# Id of some image from the dataset
IMAGE_ID = 404484
# Get an image file path
image_path = loader.load_path(IMAGE_ID)
# Plot the image
I = image_utils.imread(image_path)
plt.imshow(I)
plt.title("Image")
plt.show()
# Grid params: 5 splits along Y, and 5 along X
GRID = (5, 5)
# Create a position transformer
position_transformer = PositionSetTransformer(splits=GRID, element_ndim=3)
# Set an image size
position_transformer.fit(X=I, y=None)
# Build an image in which each pixel defines a position
position_image = position_transformer.transform(X=I)
# Plot the positional element along with the initial image
fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("Low-level position elements")
axes[1].imshow(position_image)
axes[1] = plot_position_grid(position_transformer, axes[1])
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(I)
axes[2] = plot_position_grid(position_transformer, axes[2])
axes[2].axis("off")
plt.show()
# Create an instance of query parser
parser = Parser()
# Definition of high-level positional elements
Ep_top = E("1+2+3+4+5+6+7+8+9+10")
Ep_bottom = E("16+17+18+19+20+21+22+23+24+25")
Ep_left = E("1+2+6+7+11+12+16+17+21+22")
Ep_right = E("4+5+9+10+14+15+19+20+24+25")
Ep_center = E("7+8+9+12+13+14+17+18+19")
Eps = [
("top", Ep_top),
("bottom", Ep_bottom),
("left", Ep_left),
("right", Ep_right),
("center", Ep_center)
]
# Sets of high-level positional elements (they will be used for the Evaluator below)
Eps_set = { name: parser.parse_set(Ep.value) for name, Ep in Eps}
Eps_set["center"]
{'12', '13', '14', '17', '18', '19', '7', '8', '9'}
# Plot a high-level element
Ep_set_ = Eps_set["center"]
elements_image = position_transformer.filter_elements(position_image, Ep_set_)
filtered_image = position_transformer.filter_data(position_image, I, Ep_set_)
fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("High-level position elements")
axes[1].imshow(elements_image)
axes[1] = plot_position_grid(position_transformer, axes[1], Ep_set_)
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(filtered_image)
axes[2] = plot_position_grid(position_transformer, axes[2], Ep_set_)
axes[2].get_xaxis().set_visible(False)
axes[2].get_yaxis().set_visible(False)
plt.show()
object_names = loader.load_object_names()
print("Total number of categories: {}".format(len(object_names)))
object_names[:5]
Total number of categories: 80
[(1, 'person'), (2, 'bicycle'), (3, 'car'), (4, 'motorcycle'), (5, 'airplane')]
# Create object transformer
object_transformer = COCOObjectSetTransformer(coco)
# Build object mask
object_image = object_transformer.fit_transform(X=None, y=None, ids=IMAGE_ID)
# Mask image based on segments
image__object_filtered = object_transformer.filter_data(object_image, I)
# Plot initial image, object image, and filtered image
fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("Low-level object elements")
axes[1].imshow(object_image)
axes[1] = plot_object_ids(object_image, axes[1])
# axes[1] = utils.plot_object_edges(object_transformer, axes[1])
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(I)
axes[2].imshow(image__object_filtered, alpha=0.5)
axes[2] = plot_object_ids(object_image, axes[2])
# axes[2] = utils.plot_object_edges(object_transformer, axes[2])
axes[2].axis("off")
plt.show()
object_transformer.get_element_name(extract_elements(object_image))
[('person', 1),
('dog', 18),
('potted plant', 64),
('tv', 72),
('teddy bear', 88)]
# Definition of high-level positional elements
Eo_living_beings = E("1+18")
Eo_person = E("1")
Eo_dog = E("18")
Eos = [
("living_beings", Eo_living_beings),
("person", Eo_person),
("dog", Eo_dog),
]
# Sets of high-level positional elements (they will be used for the Evaluator below)
Eos_set = { name: parser.parse_set(Eo.value) for name, Eo in Eos}
Eos_set["living_beings"]
{'1', '18'}
# Plot a high-level element
Eo_set_ = Eos_set["living_beings"]
elements_image = object_transformer.filter_elements(object_image, Eo_set_)
filtered_image = object_transformer.filter_data(object_image, I, Eo_set_)
fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("High-level object elements")
axes[1].imshow(elements_image)
axes[1] = plot_object_ids(elements_image, axes[1], Eo_set_)
# axes[1] = utils.plot_object_edges(object_transformer, axes[1], Eo_set_)
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(I)
axes[2].imshow(filtered_image, alpha=0.5)
axes[2] = plot_object_ids(elements_image, axes[2], Eo_set_)
# axes[2] = utils.plot_object_edges(object_transformer, axes[2], Eo_set_)
axes[2].axis("off")
plt.show()
object_transformer.get_element_name(Eos_set["living_beings"])
[('dog', '18'), ('person', '1')]
# Option 1
hist = create_histogram((position_image, object_image))
hist.to_dict()
{('1', '72'): 9.114583333333334e-05,
('3', '1'): 0.0037239583333333335,
('4', '1'): 0.0130078125,
('6', '72'): 0.010208333333333333,
('7', '18'): 0.0006119791666666667,
('8', '1'): 0.0004166666666666667,
('9', '1'): 0.032578125,
('10', '1'): 0.0019921875,
('10', '64'): 0.007005208333333333,
('11', '72'): 0.003203125,
('11', '88'): 0.0012109375,
('12', '18'): 0.016315104166666667,
('12', '88'): 0.0055859375,
('13', '18'): 0.013619791666666667,
('14', '1'): 0.0036067708333333334,
('14', '64'): 0.016848958333333334,
('15', '64'): 0.02421875,
('17', '18'): 0.001953125,
('17', '88'): 0.0003515625,
('18', '18'): 0.0014713541666666666,
('19', '64'): 0.0004427083333333333,
('20', '64'): 0.0014192708333333334}
# Option 2.a Merge features into a single image
feature_merger = FeatureMerger()
merged_image = feature_merger.fit_transform((position_image, object_image))
merged_image
rec.array([[( 1, 0), ( 1, 0), ( 1, 0), ..., ( 5, 0), ( 5, 0), ( 5, 0)],
[( 1, 0), ( 1, 0), ( 1, 0), ..., ( 5, 0), ( 5, 0), ( 5, 0)],
[( 1, 0), ( 1, 0), ( 1, 0), ..., ( 5, 0), ( 5, 0), ( 5, 0)],
...,
[(21, 0), (21, 0), (21, 0), ..., (25, 0), (25, 0), (25, 0)],
[(21, 0), (21, 0), (21, 0), ..., (25, 0), (25, 0), (25, 0)],
[(21, 0), (21, 0), (21, 0), ..., (25, 0), (25, 0), (25, 0)]],
dtype=[('f0', 'u1'), ('f1', '<i8')])
# Option 2.b Create a histogram
hist = create_histogram_(merged_image)
hist.to_dict()
{('1', '72'): 9.114583333333334e-05,
('3', '1'): 0.0037239583333333335,
('4', '1'): 0.0130078125,
('6', '72'): 0.010208333333333333,
('7', '18'): 0.0006119791666666667,
('8', '1'): 0.0004166666666666667,
('9', '1'): 0.032578125,
('10', '1'): 0.0019921875,
('10', '64'): 0.007005208333333333,
('11', '72'): 0.003203125,
('11', '88'): 0.0012109375,
('12', '18'): 0.016315104166666667,
('12', '88'): 0.0055859375,
('13', '18'): 0.013619791666666667,
('14', '1'): 0.0036067708333333334,
('14', '64'): 0.016848958333333334,
('15', '64'): 0.02421875,
('17', '18'): 0.001953125,
('17', '88'): 0.0003515625,
('18', '18'): 0.0014713541666666666,
('19', '64'): 0.0004427083333333333,
('20', '64'): 0.0014192708333333334}
# TODO:
# OBJ_COLOR = {str(el[0]): np.random.randint(0, 255, 3) for el in object_names}
high_level_elements = {
0: Eps_set, # positions
1: Eos_set # objects
}
evaluator = Evaluator(operations, hist, high_level_elements=high_level_elements)
POS1 = "center"
OBJ1 = "person"
POS2 = "left"
OBJ2 = "dog"
E1 = E(POS1, OBJ1)
E2 = E(POS2, OBJ2)
E1_expr = parser.parse_string(E1.value)
HE1 = evaluator.eval(E1_expr)
print("Expression for E1:\n{}".format(E1.value))
print("\nThe parsed expressino for E1 in the postfix notation:\n{}".format(E1_expr))
print("\nHistogram of E1 given the image:\n{}".format(HE1.to_dict()))
print("\nValue of presence for E1:\n{}".format(HE1.sum()))
Expression for E1:
(center,person)
The parsed expressino for E1 in the postfix notation:
[('center', 'person')]
Histogram of E1 given the image:
{('8', '1'): 0.0004166666666666667, ('14', '1'): 0.0036067708333333334, ('9', '1'): 0.032578125}
Value of presence for E1:
0.0366015625
E2_expr = parser.parse_string(E2.value)
HE2 = evaluator.eval(E2_expr)
print("Expression for E2:\n{}".format(E2.value))
print("\nThe parsed expressino for E2 in the postfix notation:\n{}".format(E2_expr))
print("\nHistogram of E2 given the image:\n{}".format(HE2.to_dict()))
print("\nValue of presence for E2:\n{}".format(HE2.sum()))
Expression for E2:
(left,dog)
The parsed expressino for E2 in the postfix notation:
[('left', 'dog')]
Histogram of E2 given the image:
{('7', '18'): 0.0006119791666666667, ('17', '18'): 0.001953125, ('12', '18'): 0.016315104166666667}
Value of presence for E2:
0.018880208333333336
# Plot histogram elements
E1_set = extract_element_set(HE1, 2)
E2_set = extract_element_set(HE2, 2)
E1_image = filter_data(I, merged_image, HE1.elements())
E2_image = filter_data(I, merged_image, HE2.elements())
fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].get_xaxis().set_visible(False)
axes[0].get_yaxis().set_visible(False)
axes[1].set_title("E1: {}".format(E1.value))
axes[1].imshow(I)
axes[1].imshow(E1_image, alpha=0.8)
axes[1] = plot_position_grid(position_transformer, axes[1], E1_set[0])
axes[1] = plot_object_ids(elements_image, axes[1], E1_set[1])
axes[1].get_xaxis().set_visible(False)
axes[1].get_yaxis().set_visible(False)
axes[2].set_title("E2: {}".format(E2.value))
axes[2].imshow(I)
axes[2].imshow(E2_image, alpha=0.8)
axes[2] = plot_position_grid(position_transformer, axes[2], E2_set[0])
axes[2] = plot_object_ids(elements_image, axes[2], E2_set[1])
axes[2].get_xaxis().set_visible(False)
axes[2].get_yaxis().set_visible(False)
plt.show()
# Expression with union
E_union = E1 + E2
# Parsed expression
E_union_expr = parser.parse_string(E_union.value)
# Calculate histogram value
HE_union = evaluator.eval(E_union_expr)
print("Expression for E_union:\n{}".format(E_union))
print("\nThe parsed expression for E_union in the postfix notation:\n{}".format(E_union_expr))
print("\nHistogram of E_union given the image:\n{}".format(HE_union.to_dict()))
print("\nValue of presence for E_union:\n{}".format(HE_union.sum()))
Expression for E_union:
((center,person)+(left,dog))
The parsed expression for E_union in the postfix notation:
[('center', 'person'), ('left', 'dog'), '+']
Histogram of E_union given the image:
{('8', '1'): 0.0004166666666666667, ('14', '1'): 0.0036067708333333334, ('7', '18'): 0.0006119791666666667, ('17', '18'): 0.001953125, ('9', '1'): 0.032578125, ('12', '18'): 0.016315104166666667}
Value of presence for E_union:
0.05548177083333333
# Extract ids of non-zero elements for each feature
E_result_set = extract_element_set(HE_union, 2)
# Plot elements and result
transformers = (position_transformer, object_transformer)
titles = ["E1: {}".format(E1), "E2: {}".format(E2), "Result: {}".format(E_union)]
show_operation_result(I, merged_image, "f1", HE1, HE2, HE_union, transformers, titles)
operation_list = [
# set operations
("union", "+", E1 + E2),
("intersection", "*", E1 * E2),
("substraction", "-", E1 - E2), # or exception, or E1.Sub(E2)
# logic operations
("and", "&", E1 & E2), # or E1.And(E2)
("or", "|", E1 | E2), # or E1.Or(E2)
("xor", "^", E1 ^ E2), # or E1.Xor(E2)
("xsubstraction", "Xsub", E1.Xsub(E2)),
]
for op_name, op_sign, op in operation_list:
E_expr = parser.parse_string(op.value)
HE = evaluator.eval(E_expr)
print("{:12}{:^12}{:10}".format("Operation", "Sign", "Result"))
print("{}".format("-"*34))
print("{:12}{:^12}{:.5f}".format(op_name, op_sign, HE.sum()))
E_result_set = extract_element_set(HE, 2)
titles = ["E1: {}".format(E1), "E2: {}".format(E2), "Result: {}".format(op)]
show_operation_result(I, merged_image, "f1", HE1, HE2, HE, transformers, titles)
Operation Sign Result ---------------------------------- union + 0.05548
Operation Sign Result ---------------------------------- intersection * 0.00000
Operation Sign Result ---------------------------------- substraction - 0.03660
Operation Sign Result ---------------------------------- and & 0.01888
Operation Sign Result ---------------------------------- or | 0.05548
Operation Sign Result ---------------------------------- xor ^ 0.03660
Operation Sign Result ---------------------------------- xsubstraction Xsub 0.00000
# Load histograms of coco images if exist, otherwise transform images to histograms and return
hists = load_coco_histograms(coco)
hists[:5]
[(397133, <himpy.histogram.Histogram1D at 0x7f4b941cb3c8>), (37777, <himpy.histogram.Histogram1D at 0x7f4b941d3630>), (252219, <himpy.histogram.Histogram1D at 0x7f4b9427fcc0>), (87038, <himpy.histogram.Histogram1D at 0x7f4b9427fd68>), (174482, <himpy.histogram.Histogram1D at 0x7f4b9427f320>)]
# Initialize a search engine
search_engine = SearchEngine(hists, parser, evaluator)
TOP_N = 20
# Define your query
query = E("left", "dog") & E("center", "person")
# Retrieve images using the query
ranked_images = search_engine.retrieve(query, topN=TOP_N)
print("Total retrieved images:", len(ranked_images))
ranked_images[:5]
Total retrieved images: 20
[(432553, 0.10005122950819673), (236166, 0.09953418427230046), (107226, 0.08212565104166666), (246454, 0.07708740234375), (117525, 0.072228)]
# Compose paths of retrieved images
image_paths = [
loader.load_path(image_id)
for image_id, _ in ranked_images
]
image_paths[:1]
['datasets/coco/val2017/000000432553.jpg']
# Show top ranked images
show_retrieved_images(ranked_images, image_paths, limit=TOP_N, title="Query: {}".format(query.value))
# Take a sample image and its histogram from the list of histograms that were created previously
sample_image_id = hists[4][0]
sample_hist = hists[4][1]
# Show the sample image
sample_image_path = loader.load_path(sample_image_id)
I = image_utils.imread(sample_image_path)
plt.imshow(I)
plt.title("Image")
plt.show()
# Retrieve images similar to the sample
ranked_images__sample = search_engine.retrieve(sample_hist, topN=TOP_N)
print("Total retrieved images:", len(ranked_images__sample))
ranked_images__sample[:5]
Total retrieved images: 20
[(174482, 0.36078849871134017), (109055, 0.2703729730455326), (492937, 0.25539785193227954), (274687, 0.19363381926546394), (101762, 0.18673747583762884)]
# Compose paths of retrieved images
image_paths__sample = [
loader.load_path(image_id)
for image_id, _ in ranked_images__sample
]
# Show top ranked images
show_retrieved_images(
ranked_images__sample,
image_paths__sample,
limit=TOP_N,
title="Query: {}".format("Sample Image")
)